// SPDX-License-Identifier: GPL-2.0

#include <linux/module.h>
#include <linux/init.h>
#include <linux/device.h>
#include <linux/err.h>
#include <linux/fs.h>
#include <linux/cdev.h>
#include <linux/dma-mapping.h>
#include <linux/poll.h>
#include <linux/mutex.h>
#include <linux/kref.h>
#include <linux/compat.h>
#include <linux/sched.h>
#include <linux/semaphore.h>
#include <linux/slab.h>

#include <linux/uaccess.h>
#include <trace/hooks/rdma_pangea.h>

#include "core_priv.h"
#include "hw_compat.h"
#include <rdma/ib_user_uhw.h>
#include <rdma/ib.h>

#define INIT_IB_HW_EVENT_HANDLER(_ptr, _device, _handler)  \
	do {    \
		(_ptr)->device  = (_device);  \
		(_ptr)->handler = (_handler);  \
		INIT_LIST_HEAD(&(_ptr)->list);  \
	} while (0)

static dev_t dynamic_hw_compat_dev;
static struct class *hw_compat_class;
static DECLARE_BITMAP(dev_map, IB_HW_COMPAT_MAX_DEVICES);

static int ib_hw_compat_add_one(struct ib_device *device);
static void ib_hw_compat_remove_one(struct ib_device *device, void *client_data);

static void ib_hw_compat_release_dev(struct kobject *kobj)
{
	struct ib_hw_compat_device *dev =
		container_of(kobj, struct ib_hw_compat_device, kobj);

	cleanup_srcu_struct(&dev->disassociate_srcu);
	kfree(dev);
}

static struct kobj_type ib_hw_compat_dev_ktype = {
	.release = ib_hw_compat_release_dev,
};

void ib_dispatch_hw_event(struct ib_hw_event *event)
{
	unsigned long flags;
	struct ib_hw_event_handler *handler;
	struct ib_device_ex *ib_dev_ex = ib_get_device_ex(event->device);

	spin_lock_irqsave(&ib_dev_ex->hw_event_handler_lock, flags);

	list_for_each_entry(handler, &ib_dev_ex->hw_event_handler_list, list)
		handler->handler(handler, event);

	spin_unlock_irqrestore(&ib_dev_ex->hw_event_handler_lock, flags);
}
EXPORT_SYMBOL(ib_dispatch_hw_event);

static void ib_register_hw_event_handler(struct ib_hw_event_handler *event_handler)
{
	unsigned long flags;
	struct ib_device_ex *ib_dev_ex = NULL;

	ib_dev_ex = ib_get_device_ex(event_handler->device);

	spin_lock_irqsave(&ib_dev_ex->hw_event_handler_lock, flags);
	list_add_tail(&event_handler->list, &ib_dev_ex->hw_event_handler_list);
	spin_unlock_irqrestore(&ib_dev_ex->hw_event_handler_lock, flags);
}

static void ib_unregister_hw_event_handler(struct ib_hw_event_handler *event_handler)
{
	unsigned long flags;
	struct ib_device_ex *ib_dev_ex = NULL;

	ib_dev_ex = ib_get_device_ex(event_handler->device);

	spin_lock_irqsave(&ib_dev_ex->hw_event_handler_lock, flags);
	list_del(&event_handler->list);
	spin_unlock_irqrestore(&ib_dev_ex->hw_event_handler_lock, flags);
}

void ib_hw_compat_event_handler(struct ib_hw_event_handler *handler,
	struct ib_hw_event *evt)
{
	struct ib_hw_compat_event *hw_compat_evt = NULL;
	struct ib_hw_compat_file *file =
		container_of(handler, struct ib_hw_compat_file, event_handler);
	unsigned long flags;

	spin_lock_irqsave(&file->event_queue.lock, flags);
	if (file->event_queue.is_closed) {
		spin_unlock_irqrestore(&file->event_queue.lock, flags);
		return;
	}

	hw_compat_evt = kzalloc(sizeof(*hw_compat_evt) + evt->len, GFP_ATOMIC);
	if (!hw_compat_evt) {
		ofed_log_print(PANGEA_ERR, "Kzalloc evt(%d) failed, size(%zu), len(%d), dev(%s).",
			evt->event, sizeof(*hw_compat_evt), evt->len, evt->device->name);
		spin_unlock_irqrestore(&file->event_queue.lock, flags);
		return;
	}

	hw_compat_evt->len = evt->len;
	hw_compat_evt->type = evt->event;
	if (evt->len)
		memcpy(hw_compat_evt->data, evt->data, evt->len);

	list_add_tail(&hw_compat_evt->list, &file->event_queue.event_list);
	spin_unlock_irqrestore(&file->event_queue.lock, flags);
	wake_up_interruptible(&file->event_queue.poll_wait);
}

static ssize_t ib_uhw_send_msg(struct ib_hw_compat_file *file,
	const char __user *inbuf, int in_len, int out_len)
{
	int ret;
	struct ib_uhw_send cmd;
	struct ib_device *ib_dev = NULL;
	struct ib_device_ex *ib_dev_ex = NULL;

	uint8_t *buf = NULL;

	ib_dev = srcu_dereference(file->device->ib_dev, &file->device->disassociate_srcu);
	if (!ib_dev)
		return -EIO;

	if (copy_from_user(&cmd, inbuf, sizeof(cmd))) {
		ofed_log_print(PANGEA_ERR, "Copy from user failed.");
		return -EFAULT;
	}

	ib_dev_ex = ib_get_device_ex(ib_dev);
	if (!ib_dev_ex && !ib_dev_ex->ops.send_conn_msg) {
		ofed_log_print(PANGEA_ERR, "Not support send conn msg, dev(%s).",
			ib_dev->name);
		return -EFAULT;
	}

	buf = kzalloc(cmd.private_data_len, GFP_KERNEL);
	if (!buf) {
		ofed_log_print(PANGEA_ERR, "Failed to malloc kv data, len(%d), dev(%s).",
			cmd.private_data_len, ib_dev->name);
		return -EFAULT;
	}
	if (copy_from_user(buf, u64_to_user_ptr(cmd.data), cmd.private_data_len)) {
		ofed_log_print(PANGEA_ERR, "Copy from user failed, len(%d), dev(%s).",
			cmd.private_data_len, ib_dev->name);
		kfree(buf);
		return -EFAULT;
	}

	ret = ib_dev_ex->ops.send_conn_msg(ib_dev, cmd.hostid, buf, cmd.private_data_len);
	if (ret)
		ofed_log_print(PANGEA_ERR,
			"Failed to send conn msg to host(%u), ret(%d), ib_dev(%pK), dev(%s).",
			cmd.hostid, ret, ib_dev, ib_dev->name);

	kfree(buf);
	return ret;
}

static ssize_t ib_uhw_get_event(struct ib_hw_compat_file *file,
	const char __user *inbuf, int in_len, int out_len)
{
	struct ib_uhw_get cmd;
	struct ib_uhw_get_resp *resp = NULL;
	struct ib_hw_compat_event *event = NULL;
	int ret = 0;

	if (out_len < sizeof(*resp)) {
		ofed_log_print(PANGEA_ERR, "Out_len(%d), resp(%zu).", out_len, sizeof(resp));
		return -ENOSPC;
	}

	if (copy_from_user(&cmd, inbuf, sizeof(cmd))) {
		ofed_log_print(PANGEA_ERR, "Failed to copy from user.");
		return -EFAULT;
	}

	resp = kzalloc(out_len, GFP_KERNEL);
	if (!resp) {
		ofed_log_print(PANGEA_ERR, "Failed to malloc resp:%d.", out_len);
		return -ENOMEM;
	}

	spin_lock_irq(&file->event_queue.lock);
	while (list_empty(&file->event_queue.event_list)) {
		spin_unlock_irq(&file->event_queue.lock);

		if (file->filp->f_flags & O_NONBLOCK) {
			ofed_log_print(PANGEA_ERR, "File flags(%d).", file->filp->f_flags);
			ret = -EAGAIN;
			goto out;
		}

		if (wait_event_interruptible(file->event_queue.poll_wait,
			!list_empty(&file->event_queue.event_list))) {
			ofed_log_print(PANGEA_ERR, "Failed to wait event interrupt.");
			ret = -ERESTARTSYS;
			goto out;
		}

		spin_lock_irq(&file->event_queue.lock);
	}

	event = list_entry(file->event_queue.event_list.next, struct ib_hw_compat_event, list);
	list_del(&event->list);
	spin_unlock_irq(&file->event_queue.lock);

	resp->data_len = event->len;
	resp->event = event->type;
	memcpy(resp->data, event->data, event->len);
	if (copy_to_user(u64_to_user_ptr(cmd.response), resp,
		min_t(size_t, out_len, sizeof(*resp) + event->len))) {
		ofed_log_print(PANGEA_ERR,
			"Failed to copy to user, event:%u, out_len:%d, size:%zu, event len:%d.",
			event->type, out_len, sizeof(*resp), event->len);
		ret = -EFAULT;
	}

	kfree(event);
out:
	kfree(resp);
	return ret;
}

static ssize_t ib_uhw_set_conn_sts(struct ib_hw_compat_file *file,
	const char __user *inbuf, int in_len, int out_len)
{
	int ret;
	struct ib_uhw_set_conn_sts cmd;
	struct ib_device *ib_dev = NULL;
	struct ib_device_ex *ib_dev_ex = NULL;

	ib_dev = srcu_dereference(file->device->ib_dev, &file->device->disassociate_srcu);
	if (!ib_dev)
		return -EIO;

	if (copy_from_user(&cmd, inbuf, sizeof(cmd))) {
		ofed_log_print(PANGEA_ERR, "Copy from user failed.");
		return -EFAULT;
	}

	ib_dev_ex = ib_get_device_ex(ib_dev);
	if (!ib_dev_ex && !ib_dev_ex->ops.set_conn_sts) {
		ofed_log_print(PANGEA_ERR,
			"Not support set conn sts, ib_dev(%pK), name(%s), num(%d).",
			ib_dev, ib_dev->name, cmd.num);
		return -EFAULT;
	}

	ret = ib_dev_ex->ops.set_conn_sts(ib_dev, cmd.data, cmd.num);
	if (ret) {
		ofed_log_print(PANGEA_ERR,
			"Failed to set conn sts, ret(%d), ib_dev(%pK), name(%s).",
			ret, ib_dev, ib_dev->name);
	}
	return ret;
}

static ssize_t ib_uhw_disconnect(struct ib_hw_compat_file *file,
	const char __user *inbuf, int in_len, int out_len)
{
	int ret;
	struct ib_uhw_disconnect cmd;
	struct ib_uhw_disconn_param disconn_param;
	struct ib_device *ib_dev = NULL;
	struct ib_device_ex *ib_dev_ex = NULL;

	ib_dev = srcu_dereference(file->device->ib_dev, &file->device->disassociate_srcu);
	if (!ib_dev)
		return -EIO;

	if (copy_from_user(&cmd, inbuf, sizeof(cmd))) {
		ofed_log_print(PANGEA_ERR, "Copy from user failed.");
		return -EFAULT;
	}

	ib_dev_ex = ib_get_device_ex(ib_dev);

	if (!ib_dev_ex && !ib_dev_ex->ops.disconnect_qp) {
		ret = -EOPNOTSUPP;
		ofed_log_print(PANGEA_ERR, "Driver not support disconnect qp, device(%s), ret(%d).",
			ib_dev->name, ret);
		return ret;
	}

	memcpy(&disconn_param, &cmd, sizeof(cmd));
	ret = ib_dev_ex->ops.disconnect_qp(ib_dev, &disconn_param);
	if (ret)
		ofed_log_print(PANGEA_ERR, "Send dreq failed, ret(%d), device(%s).",
			ret, ib_dev->name);
	return ret;
}

static ssize_t ib_uhw_query_qplist(struct ib_hw_compat_file *file,
	const char __user *inbuf, int in_len, int out_len)
{
	int ret = 0;
	struct ib_uhw_query_qplist cmd;
	struct ib_uhw_query_qplist_resp resp;
	struct ib_uhw_qp_list qp_list;
	struct ib_device *ib_dev = NULL;
	struct ib_device_ex *ib_dev_ex = NULL;

	ib_dev = srcu_dereference(file->device->ib_dev, &file->device->disassociate_srcu);
	if (!ib_dev)
		return -EIO;

	if (copy_from_user(&cmd, inbuf, sizeof(cmd))) {
		ofed_log_print(PANGEA_ERR, "Copy from user failed.");
		return -EFAULT;
	}

	ib_dev_ex = ib_get_device_ex(ib_dev);
	if (!ib_dev_ex && !ib_dev_ex->ops.query_qplist) {
		ofed_log_print(PANGEA_ERR,
			"Driver not support query qp list, ib_dev(%pK), name(%s), data_len(%u).",
			ib_dev, ib_dev->name, cmd.data_len);
		return -EOPNOTSUPP;
	}

	qp_list.data_len = cmd.data_len;
	qp_list.data = kzalloc(cmd.data_len, GFP_KERNEL);
	if (!qp_list.data) {
		ofed_log_print(PANGEA_ERR, "Failed to kzalloc resp len(%d), device(%s).",
			cmd.data_len, file->device->ib_dev->name);
		return -ENOMEM;
	}

	ret = ib_dev_ex->ops.query_qplist(ib_dev, &qp_list);
	if (ret) {
		ofed_log_print(PANGEA_ERR,
			"Failed to query qp list, ret(%d), ib_dev(%pK), name(%s).",
			ret, ib_dev, ib_dev->name);
		goto out;
	}

	resp.start = qp_list.start;
	resp.num = qp_list.num;

	if (copy_to_user(u64_to_user_ptr(cmd.data), qp_list.data,
		min_t(size_t, qp_list.data_len, cmd.data_len))) {
		ofed_log_print(PANGEA_ERR,
			"Failed to copy data to user, out_len(%d), data_len(%u), device(%s).",
			qp_list.data_len, cmd.data_len, file->device->ib_dev->name);
		ret = -EFAULT;
		goto out;
	}

	if (copy_to_user(u64_to_user_ptr(cmd.response), &resp,
		min_t(size_t, out_len, sizeof(resp)))) {
		ofed_log_print(PANGEA_ERR,
			"Copy resp to user failed, out_len(%d), data_len(%u), dev(%s).",
			out_len, cmd.data_len, file->device->ib_dev->name);
		ret = -EFAULT;
		goto out;
	}

out:
	kfree(qp_list.data);
	return ret;
}

static ssize_t ib_uhw_query_master_qpc(struct ib_hw_compat_file *file,
	const char __user *inbuf, int in_len, int out_len)
{
	int ret;
	struct ib_uhw_query_master_qpc cmd;
	struct ib_uhw_query_master_qpc_resp resp = {0};
	struct ib_device *ib_dev = NULL;
	struct ib_device_ex *ib_dev_ex = NULL;
	struct ib_uhw_priv_qpc_info qpc_info = {0};

	ib_dev = srcu_dereference(file->device->ib_dev, &file->device->disassociate_srcu);
	if (!ib_dev)
		return -EIO;

	if (copy_from_user(&cmd, inbuf, sizeof(cmd))) {
		ofed_log_print(PANGEA_ERR, "Copy from user failed.");
		return -EFAULT;
	}

	ib_dev_ex = ib_get_device_ex(ib_dev);
	if (!ib_dev_ex && !ib_dev_ex->ops.query_master_qpc) {
		ofed_log_print(PANGEA_ERR,
			"Driver not support query master qp, ib_dev(%pK), name(%s), qpn(%u).",
			ib_dev, ib_dev->name, cmd.qpn);
		return -EOPNOTSUPP;
	}

	ret = ib_dev_ex->ops.query_master_qpc(ib_dev, cmd.qpn, &qpc_info);
	if (ret) {
		ofed_log_print(PANGEA_ERR,
			"Failed to query master qp, ret(%d), ib_dev(%pK), name(%s), qpn(%u).",
			ret, ib_dev, ib_dev->name, cmd.qpn);
		return ret;
	}

	resp.host = qpc_info.host;
	resp.admin_qp = qpc_info.admin_qp;
	resp.qp_state = qpc_info.qp_state;

	if (copy_to_user(u64_to_user_ptr(cmd.response), &resp,
		min_t(size_t, out_len, sizeof(resp)))) {
		ofed_log_print(PANGEA_ERR, "Failed to copy to user.");
		return -EFAULT;
	}

	return 0;
}

static ssize_t ib_uhw_query_hostid(struct ib_hw_compat_file *file,
	const char __user *inbuf, int in_len, int out_len)
{
	int ret;
	unsigned int hostid;
	struct ib_uhw_query_hostid cmd;
	struct ib_uhw_query_hostid_resp resp = {0};
	struct ib_device *ib_dev = NULL;
	struct ib_device_ex *ib_dev_ex = NULL;

	ib_dev = srcu_dereference(file->device->ib_dev, &file->device->disassociate_srcu);
	if (!ib_dev)
		return -EIO;

	if (copy_from_user(&cmd, inbuf, sizeof(cmd))) {
		ofed_log_print(PANGEA_ERR, "Copy from user failed.");
		return -EFAULT;
	}

	ib_dev_ex = ib_get_device_ex(ib_dev);
	if (!ib_dev_ex && !ib_dev_ex->ops.query_hostid) {
		ofed_log_print(PANGEA_ERR,
			"Driver not support query hostid, ib_dev(%pK), device name(%s).",
			ib_dev, ib_dev->name);
		return -EOPNOTSUPP;
	}

	ret = ib_dev_ex->ops.query_hostid(ib_dev, &hostid);
	if (ret) {
		ofed_log_print(PANGEA_ERR,
			"Failed to query hostid, ret(%d), ib_dev(%pK), device name(%s).",
			ret, ib_dev, ib_dev->name);
		return ret;
	}

	resp.hostid = hostid;

	if (copy_to_user(u64_to_user_ptr(cmd.response), &resp,
		min_t(size_t, out_len, sizeof(resp)))) {
		ofed_log_print(PANGEA_ERR, "Failed to copy to user.");
		return -EFAULT;
	}

	return 0;
}

static ssize_t ib_uhw_ready_to_use(struct ib_hw_compat_file *file,
	const char __user *inbuf, int in_len, int out_len)
{
	int ret;
	struct ib_device *ib_dev = NULL;
	struct ib_device_ex *ib_dev_ex = NULL;

	ib_dev = srcu_dereference(file->device->ib_dev, &file->device->disassociate_srcu);
	if (!ib_dev)
		return -EIO;

	ib_dev_ex = ib_get_device_ex(ib_dev);
	if (!ib_dev_ex && !ib_dev_ex->ops.ready_to_use) {
		ofed_log_print(PANGEA_ERR,
			"Driver not support ready to use, ib_dev(%pK), name(%s).",
			ib_dev, ib_dev->name);
		return -EOPNOTSUPP;
	}

	ret = ib_dev_ex->ops.ready_to_use(ib_dev);
	if (ret) {
		ofed_log_print(PANGEA_ERR, "Ready to use failed, ret(%d), ib_dev(%pK), name(%s).",
			ret, ib_dev, ib_dev->name);
		return ret;
	}

	return 0;
}

static ssize_t ib_uhw_set_shard_cfg(struct ib_hw_compat_file *file,
	const char __user *inbuf, int in_len, int out_len)
{
	struct ib_uhw_shard_cfg cmd;
	int ret;
	struct ib_device *ib_dev = NULL;
	struct ib_device_ex *ib_dev_ex = NULL;
	uint8_t *entry_data = NULL;

	ib_dev = srcu_dereference(file->device->ib_dev, &file->device->disassociate_srcu);
	if (!ib_dev)
		return -EIO;

	if (copy_from_user(&cmd, inbuf, sizeof(cmd))) {
		ofed_log_print(PANGEA_ERR, "Copy from user failed.");
		return -EFAULT;
	}

	ib_dev_ex = ib_get_device_ex(ib_dev);
	if (!ib_dev_ex && !ib_dev_ex->ops.set_shard_cfg) {
		ofed_log_print(PANGEA_ERR, "Driver not support set dd cfg, ib_dev(%pK), name(%s).",
			ib_dev, ib_dev->name);
		return -EFAULT;
	}

	entry_data = kzalloc(cmd.entry_data_len, GFP_KERNEL);
	if (!entry_data) {
		ofed_log_print(PANGEA_ERR, "Failed to malloc shard data, len(%d).",
			cmd.entry_data_len);
		return -EFAULT;
	}
	if (copy_from_user(entry_data, u64_to_user_ptr(cmd.entry_data), cmd.entry_data_len)) {
		ofed_log_print(PANGEA_ERR, "Copy from user failed, len(%d).", cmd.entry_data_len);
		kfree(entry_data);
		return -EFAULT;
	}

	ret = ib_dev_ex->ops.set_shard_cfg(ib_dev, cmd.op, entry_data, cmd.entry_data_len);
	if (ret)
		ofed_log_print(PANGEA_ERR,
			"Failed to set dd cfg, op(%d), ret(%d), ib_dev(%pK), name(%s), len(%d).",
			cmd.op, ret, ib_dev, ib_dev->name, cmd.entry_data_len);

	kfree(entry_data);
	return ret;
}

static ssize_t ib_uhw_set_dd_cfg(struct ib_hw_compat_file *file,
	const char __user *inbuf, int in_len, int out_len)
{
	struct ib_uhw_dd_cfg cmd;
	int ret;
	struct ib_device *ib_dev = NULL;
	struct ib_device_ex *ib_dev_ex = NULL;
	uint8_t *dd_data = NULL;

	ib_dev = srcu_dereference(file->device->ib_dev, &file->device->disassociate_srcu);
	if (!ib_dev)
		return -EIO;

	if (copy_from_user(&cmd, inbuf, sizeof(cmd))) {
		ofed_log_print(PANGEA_ERR, "Copy from user failed.");
		return -EFAULT;
	}

	ib_dev_ex = ib_get_device_ex(ib_dev);
	if (!ib_dev_ex && !ib_dev_ex->ops.set_dd_cfg) {
		ofed_log_print(PANGEA_ERR, "Driver not support set dd cfg, ib_dev(%pK), name(%s).",
			ib_dev, ib_dev->name);
		return -EFAULT;
	}

	if (cmd.op == 1) {
		ret = ib_dev_ex->ops.set_dd_cfg(ib_dev, cmd.op,
					cmd.dd_id, 0, cmd.data_len, cmd.default_ctrl);
		if (ret)
			ofed_log_print(PANGEA_ERR,
				"Failed to set dd cfg, ret(%d), ib_dev(%pK), name(%s), len(%d).",
				ret, ib_dev, ib_dev->name, cmd.data_len);
		return ret;
	}

	dd_data = kzalloc(cmd.data_len, GFP_KERNEL);
	if (!dd_data) {
		ofed_log_print(PANGEA_ERR, "Failed to malloc dd data, len(%d).", cmd.data_len);
		return -EFAULT;
	}
	if (copy_from_user(dd_data, u64_to_user_ptr(cmd.data), cmd.data_len)) {
		ofed_log_print(PANGEA_ERR, "Copy from user failed, len(%d).", cmd.data_len);
		kfree(dd_data);
		return -EFAULT;
	}

	ret = ib_dev_ex->ops.set_dd_cfg(ib_dev, cmd.op,
					cmd.dd_id, dd_data, cmd.data_len, cmd.default_ctrl);
	if (ret)
		ofed_log_print(PANGEA_ERR,
			"Failed to set dd cfg, ret(%d), ib_dev(%pK), name(%s), len(%d).",
			ret, ib_dev, ib_dev->name, cmd.data_len);

	kfree(dd_data);
	return ret;
}

static ssize_t ib_uhw_switch_io(struct ib_hw_compat_file *file,
	const char __user *inbuf, int in_len, int out_len)
{
	int ret;
	struct ib_device *ib_dev = NULL;
	struct ib_device_ex *ib_dev_ex = NULL;

	ib_dev = srcu_dereference(file->device->ib_dev, &file->device->disassociate_srcu);
	if (!ib_dev)
		return -EIO;

	ib_dev_ex = ib_get_device_ex(ib_dev);
	if (!ib_dev_ex && !ib_dev_ex->ops.switch_io) {
		ofed_log_print(PANGEA_ERR, "Driver not support switch io, ib_dev(%pK), name(%s).",
			ib_dev, ib_dev->name);
		return -EOPNOTSUPP;
	}

	ret = ib_dev_ex->ops.switch_io(ib_dev);
	if (ret) {
		ofed_log_print(PANGEA_ERR, "Switch io failed, ret(%d), ib_dev(%pK), name(%s).",
			ret, ib_dev, ib_dev->name);
		return ret;
	}
	return 0;
}

static ssize_t ib_uhw_rsvd1(struct ib_hw_compat_file *file,
	const char __user *inbuf, int in_len, int out_len)
{
	return -EOPNOTSUPP;
}

static ssize_t (*uhw_cmd_table[])(struct ib_hw_compat_file *file,
	const char __user *inbuf, int in_len, int out_len) = {
	[IB_UHW_CMD_SEND_MSG] = ib_uhw_send_msg,
	[IB_UHW_CMD_GET_EVENT] = ib_uhw_get_event,
	[IB_UHW_CMD_SET_CONN_STS] = ib_uhw_set_conn_sts,
	[IB_UHW_CMD_DISCONNECT] = ib_uhw_disconnect,
	[IB_UHW_CMD_QUERY_QPLIST] = ib_uhw_query_qplist,
	[IB_UHW_CMD_QUERY_MASTER_QPC] = ib_uhw_query_master_qpc,
	[IB_UHW_CMD_QUERY_HOSTID] = ib_uhw_query_hostid,
	[IB_UHW_CMD_READY_TO_USE] = ib_uhw_ready_to_use,
	[IB_UHW_CMD_SET_DD_CFG] = ib_uhw_set_dd_cfg,
	[IB_UHW_CMD_SET_SHARD_CFG] = ib_uhw_set_shard_cfg,
	[IB_UHW_CMD_SWITCH_IO] = ib_uhw_switch_io,
	[IB_UHW_CMD_RSVD1] = ib_uhw_rsvd1,
};

static ssize_t ib_hw_compat_write(struct file *filp, const char __user *buf,
	size_t len, loff_t *pos)
{
	ssize_t ret, srcu_key;
	struct ib_hw_compat_file *file  = filp->private_data;
	struct ib_uhw_abi_hdr hdr = { 0 };

	if (!ib_safe_file_access(filp)) {
		ofed_log_print(PANGEA_ERR,
			"Process %d (%s) changed security contexts after opening fd, this is not allowed.",
			task_tgid_vnr(current), current->comm);
		return -EACCES;
	}

	if (len < sizeof(hdr)) {
		ofed_log_print(PANGEA_ERR, "Invalid hdr len(%zu).", len);
		return -EINVAL;
	}

	if (copy_from_user(&hdr, buf, sizeof(hdr))) {
		ofed_log_print(PANGEA_ERR, "Copy from user failed.");
		return -EFAULT;
	}

	if (hdr.cmd >= ARRAY_SIZE(uhw_cmd_table)) {
		ofed_log_print(PANGEA_ERR, "Invalid cmd(%d).", hdr.cmd);
		return -EINVAL;
	}

	if ((hdr.in + sizeof(hdr)) > len) {
		ofed_log_print(PANGEA_ERR, "Invalid len(%zu), in(%u).", len, hdr.in);
		return -EINVAL;
	}

	srcu_key = srcu_read_lock(&file->device->disassociate_srcu);
	ret = uhw_cmd_table[hdr.cmd](file, buf + sizeof(hdr), hdr.in, hdr.out);
	if (!ret)
		ret = len;

	srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
	return ret;
}


static __poll_t ib_hw_compat_poll(struct file *filp, struct poll_table_struct *wait)
{
	struct ib_hw_compat_file *file = filp->private_data;
	__poll_t mask = 0;

	poll_wait(filp, &file->event_queue.poll_wait, wait);

	spin_lock_irq(&file->event_queue.lock);
	if (!list_empty(&file->event_queue.event_list))
		mask = EPOLLIN | EPOLLRDNORM;
	spin_unlock_irq(&file->event_queue.lock);

	return mask;
}

static void ib_hw_init_event_queue(struct ib_hw_event_queue *event_queue, struct ib_device *ib_dev)
{
	spin_lock_init(&event_queue->lock);
	INIT_LIST_HEAD(&event_queue->event_list);
	init_waitqueue_head(&event_queue->poll_wait);
	event_queue->is_closed = 0;
}

static int ib_hw_compat_open(struct inode *inode, struct file *filp)
{
	struct ib_hw_compat_device *dev = NULL;
	struct ib_hw_compat_file *file = NULL;
	struct ib_device *ib_dev = NULL;
	int ret, srcu_key;

	ofed_log_print(PANGEA_INFO, "Normal open hw compat filp(%pK).", filp);
	dev = container_of(inode->i_cdev, struct ib_hw_compat_device, cdev);

	srcu_key = srcu_read_lock(&dev->disassociate_srcu);
	mutex_lock(&dev->list_mutex);
	ib_dev = srcu_dereference(dev->ib_dev, &dev->disassociate_srcu);
	if (!ib_dev) {
		ret = -EIO;
		goto err;
	}

	file = kzalloc(sizeof(*file), GFP_KERNEL);
	if (!file) {
		ret = -ENOMEM;
		goto err;
	}

	ib_hw_init_event_queue(&file->event_queue, ib_dev);
	file->device = dev;
	kref_init(&file->ref);
	filp->private_data = file;
	file->filp = filp;
	kobject_get(&dev->kobj);
	list_add_tail(&file->list, &dev->file_list);

	INIT_IB_HW_EVENT_HANDLER(&file->event_handler, ib_dev, ib_hw_compat_event_handler);
	ib_register_hw_event_handler(&file->event_handler);
	/* At that point event file stuff was fully set */

	mutex_unlock(&dev->list_mutex);
	srcu_read_unlock(&dev->disassociate_srcu, srcu_key);

	return nonseekable_open(inode, filp);
err:
	mutex_unlock(&dev->list_mutex);
	srcu_read_unlock(&dev->disassociate_srcu, srcu_key);
	return ret;
}

static int ib_hw_compat_close(struct inode *inode, struct file *filp)
{
	struct ib_hw_compat_file *file = filp->private_data;
	struct ib_hw_compat_device *dev = file->device;
	struct ib_hw_compat_event *entry = NULL;
	struct ib_hw_compat_event *tmp = NULL;
	int closed_already = 0;

	ofed_log_print(PANGEA_INFO, "Normal close filp(%pK).", filp);

	mutex_lock(&file->device->list_mutex);
	spin_lock_irq(&file->event_queue.lock);
	closed_already = file->event_queue.is_closed;
	file->event_queue.is_closed = 1;
	list_for_each_entry_safe(entry, tmp, &file->event_queue.event_list, list) {
		kfree(entry);
	}
	spin_unlock_irq(&file->event_queue.lock);

	if (!closed_already) {
		list_del(&file->list);
		ib_unregister_hw_event_handler(&file->event_handler);
	}
	mutex_unlock(&file->device->list_mutex);

	kfree(file);
	kobject_put(&dev->kobj);
	return 0;
}

static const struct file_operations hw_compat_fops = {
	.owner = THIS_MODULE,
	.open = ib_hw_compat_open,
	.release = ib_hw_compat_close,
	.write = ib_hw_compat_write,
	.poll = ib_hw_compat_poll,
	.llseek	 = no_llseek,
};

static CLASS_ATTR_STRING(abi_version, 0444,
	__stringify(IB_USER_VERBS_ABI_VERSION));

static struct ib_client hw_compat_client = {
	.name   = "hw_compat",
	.add    = ib_hw_compat_add_one,
	.remove = ib_hw_compat_remove_one
};

static int ib_hw_compat_add_one(struct ib_device *device)
{
	int devnum;
	dev_t base;
	int ret;
	struct ib_hw_compat_device *hw_compat_dev = NULL;

	ofed_log_print(PANGEA_INFO, "Begin add hw compat device, ib_device(%pK), name(%s).",
			device, device->name);

	hw_compat_dev = kzalloc(sizeof(*hw_compat_dev), GFP_KERNEL);
	if (!hw_compat_dev)
		return -ENOMEM;

	if (init_srcu_struct(&hw_compat_dev->disassociate_srcu)) {
		kfree(hw_compat_dev);
		return -ENOMEM;
	}

	kobject_init(&hw_compat_dev->kobj, &ib_hw_compat_dev_ktype);
	mutex_init(&hw_compat_dev->list_mutex);
	INIT_LIST_HEAD(&hw_compat_dev->file_list);

	devnum = find_first_zero_bit(dev_map, IB_HW_COMPAT_MAX_DEVICES);
	if (devnum >= IB_HW_COMPAT_MAX_DEVICES) {
		ret = -EDOM;
		goto err;
	}
	hw_compat_dev->devnum = devnum;
	set_bit(devnum, dev_map);

	base = dynamic_hw_compat_dev + devnum;
	rcu_assign_pointer(hw_compat_dev->ib_dev, device);
	cdev_init(&hw_compat_dev->cdev, &hw_compat_fops);
	hw_compat_dev->cdev.owner = THIS_MODULE;
	cdev_set_parent(&hw_compat_dev->cdev, &hw_compat_dev->kobj);
	kobject_set_name(&hw_compat_dev->cdev.kobj, "compat_%s",
		hw_compat_dev->ib_dev->name);
	ret = cdev_add(&hw_compat_dev->cdev, base, 1);
	if (ret)
		goto err_cdev;

	hw_compat_dev->dev = device_create(hw_compat_class, device->dev.parent,
		hw_compat_dev->cdev.dev, hw_compat_dev,
		"compat_%s", hw_compat_dev->ib_dev->name);
	ret = IS_ERR(hw_compat_dev->dev);
	if (ret)
		goto err_cdev;

	ib_set_client_data(device, &hw_compat_client, hw_compat_dev);
	ofed_log_print(PANGEA_INFO, "Add hw compat device done, ib_device(%pK), name(%s).",
			device, device->name);
	return 0;

err_cdev:
	cdev_del(&hw_compat_dev->cdev);
	clear_bit(devnum, dev_map);
err:
	kobject_put(&hw_compat_dev->kobj);
	return ret;
}

static void ib_hw_compat_remove_one(struct ib_device *device, void *client_data)
{
	struct ib_hw_compat_device *hw_compat_dev = client_data;
	struct ib_hw_compat_file *file = NULL;

	ofed_log_print(PANGEA_INFO, "Begin remove hw_compat device, ib_device(%pK), name(%s).",
			device, device->name);
	if (!hw_compat_dev) {
		ofed_log_print(PANGEA_ERR, "Hw_compat device is null.");
		return;
	}

	dev_set_drvdata(hw_compat_dev->dev, NULL);
	device_destroy(hw_compat_class, hw_compat_dev->cdev.dev);
	cdev_del(&hw_compat_dev->cdev);
	clear_bit(hw_compat_dev->devnum, dev_map);

	rcu_assign_pointer(hw_compat_dev->ib_dev, NULL);
	synchronize_srcu(&hw_compat_dev->disassociate_srcu);
	mutex_lock(&hw_compat_dev->list_mutex);
	hw_compat_dev->ib_dev = NULL;
	while (!(list_empty(&hw_compat_dev->file_list))) {
		file = list_first_entry(&hw_compat_dev->file_list, struct ib_hw_compat_file, list);
		spin_lock_irq(&file->event_queue.lock);
		file->event_queue.is_closed = 1;
		spin_unlock_irq(&file->event_queue.lock);
		list_del(&file->list);
		ib_unregister_hw_event_handler(&file->event_handler);
	}
	mutex_unlock(&hw_compat_dev->list_mutex);

	kobject_put(&hw_compat_dev->kobj);

	ofed_log_print(PANGEA_INFO, "Remove hw_compat device done, ib_device(%pK), name(%s).",
			device, device->name);
}

static char *hw_compat_devnode(struct device *dev, umode_t *mode)
{
	if (mode)
		*mode = 0666;

	return kasprintf(GFP_KERNEL, "infiniband/%s", dev_name(dev));
}

static int __init ib_hw_compat_init(void)
{
	int ret;

	ret = alloc_chrdev_region(&dynamic_hw_compat_dev, 0,
		IB_HW_COMPAT_NUM_DYNAMIC_MINOR, "infiniband_hw_compat");
	if (ret) {
		ofed_log_print(PANGEA_ERR, "Failed to register dynamic device number.");
		goto out;
	}

	hw_compat_class = class_create(THIS_MODULE, "infiniband_hw_compat");
	if (IS_ERR(hw_compat_class)) {
		ofed_log_print(PANGEA_ERR, "Failed to create class infiniband_hw_compat.");
		goto out_chrdev;
	}
	hw_compat_class->devnode = hw_compat_devnode;

	ret = class_create_file(hw_compat_class, &class_attr_abi_version.attr);
	if (ret) {
		ofed_log_print(PANGEA_ERR, "Failed to create abi_version attribute.");
		goto out_class;
	}

	ret = ib_register_client(&hw_compat_client);
	if (ret) {
		ofed_log_print(PANGEA_ERR, "Failed to register client.");
		goto out_class;
	}

	return 0;

out_class:
	class_destroy(hw_compat_class);
out_chrdev:
	unregister_chrdev_region(dynamic_hw_compat_dev, IB_HW_COMPAT_NUM_DYNAMIC_MINOR);
out:
	return ret;
}

static void __exit ib_hw_compat_cleanup(void)
{
	ib_unregister_client(&hw_compat_client);
	class_destroy(hw_compat_class);
	unregister_chrdev_region(dynamic_hw_compat_dev, IB_HW_COMPAT_NUM_DYNAMIC_MINOR);
}

MODULE_LICENSE("Dual BSD/GPL");
module_init(ib_hw_compat_init);
module_exit(ib_hw_compat_cleanup);
